from pyspark import SparkConf, SparkContext
import os

os.environ['PYSPARK_PYTHON'] = r"D:\Python\Python3107\python.exe"
conf = SparkConf().setMaster("local[*]").setAppName("test_spark")
sc = SparkContext(conf=conf)

rdd = sc.parallelize([1, 1, 2, 2, 3, 4, 5, 5])
print(rdd.distinct().collect())